package com.snail.tools;

import org.apache.http.HttpEntity;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

import java.io.IOException;

/**
 * @Author 【swg】.
 * @Date 2017/12/22 10:06
 * @DESC
 * @CONTACT 317758022@qq.com
 */
public class getHTML {
    private static String getH(String url) throws ClientProtocolException, IOException {
        //控制台输出日志，这样每条访问的URL都可以在控制台上看到访问情况
        System.out.println("正在解析" + url);

 /*
  * 以下内容为HttpClient建立连接的一般用法
  * 使用HttpClient建立客户端
  * 使用get方法访问指定URL
  * 获得应答
  * */

        CloseableHttpClient client = HttpClients.createDefault();
        HttpGet get = new HttpGet(url);
        CloseableHttpResponse response = client.execute(get);

 /*
  * 以下内容为将HTML内容转化为String
  * 获得应答体
  * 将应答体转为String格式，此处使用了EntityUtils中的toString方法，编码格式设置为"utf-8"
  * 完成后关闭客户端与应答
  * */
        HttpEntity entity = response.getEntity();
        String content;
        if (entity != null) {
            content = EntityUtils.toString(entity, "utf-8");
            client.close();
            response.close();
            return content;
        } else
            return null;
    }
    public static void urlControl(String baseurl, int page) throws ClientProtocolException, IOException {
        //设置当前页count
        int count = 1;
        //如果当前页小于想要爬取的页数则执行
        while (count < page) {
            //实际访问的URL为不变的URL值拼接上URL变化的值
            String u = baseurl + (2 * count - 1) + "&click=0";
            //此处调用ParseHTML类中的方法对URL中的HTML页面进行处理，后面详细介绍该类
            String content = ParseHTML.parse(getH(u)).toString();
            //此处调用WriteTo类中的方法对解析出来的内容写入到本地，后面详细介绍该类
            WriteTo.writeto(content);
            count++;
        }
    }

    public static void main(String[] args) throws IOException {
        String baseurl = "https://search.jd.com/Search?keyword=%e4%b8%89%e6%98%9f%e6%89%8b%e6%9c%ba&enc="
                + "utf-8&qrst=1&rt=1&stop=1&vt=2&cid2=653&cid3=655&page=";
        int page = 5;//设置爬取页数
        getHTML.urlControl(baseurl, page);
    }
}
